#!/usr/bin/python3
# -*- coding: utf-8; tab-width: 4; indent-tabs-mode: t -*-

import os
import sys
import time
import glob
import shutil
import random
import requests
import subprocess
import htmllistparse
import elftools.elf.elffile
import elftools.common.exceptions
from datetime import datetime
import mirrors.plugin


class Main:

    def __init__(self, sock):
        self.URL = "https://de.linuxgame.cn"

        self.EXT_INFO_DICT = {
            ".appimage": Util.isAppImageFileValid,
            ".AppImage": Util.isAppImageFileValid,            # it sucks that ...
            ".tar.gz": Util.isTarGzFileValid,
            ".tgz": Util.isTarGzFileValid,
            ".tar.bz2": Util.isTarBz2FileValid,
            ".tar.xz": Util.isTarXzFileValid,
            ".7z": Util.is7zFileValid,
            ".zip": Util.isZipFileValid,
            ".jar": Util.isZipFileValid,
        }

        self.sock = sock
        self.persist = mirrors.plugin.params["persist"]
        self.dataDir = mirrors.plugin.params["storage-file"]["data-directory"]
        self.p = InfoPrinter()

    def run(self):
        progressStage1 = 20
        progressStage2 = 70
        progressStage3 = 10

        # get file info list
        fileInfoList = []
        getFileInfoListException = None
        self.p.print("Get file info list:")
        self.p.incIndent()
        try:
            self._get_file_info_list(fileInfoList)
        except requests.exceptions.RequestException as e:
            getFileInfoListException = e
        finally:
            self.sock.progress_changed(progressStage1)
            self.p.decIndent()

        # download files
        self.p.print("Downloading files:")
        self.p.incIndent()
        try:
            for i in range(0, len(fileInfoList)):
                fileUrl, fileDatetime, localFn, localFullFn = fileInfoList[i]
                fileDatetime = fileDatetime.timestamp()
                if os.path.exists(localFullFn):
                    if os.path.getmtime(localFullFn) >= fileDatetime:                                   # we assume the timezone of the time recorded on filesystem is UTC
                        if self._check_file_integrity(localFullFn):                                     # there's no size information on website, so we have to do integrity checking
                            self.p.print("File \"%s\" is up to date." % (localFn))
                            continue
                self.p.print("Downloading \"%s\"..." % (localFn))
                Util.ensureDir(os.path.dirname(localFullFn))
                Util.downloadFile(fileUrl, localFullFn)
                os.utime(localFullFn, (fileDatetime, fileDatetime))
                self.sock.progress_changed(progressStage1 + progressStage2 * (i + 1) // len(fileInfoList))
        finally:
            self.p.decIndent()

        # remove old local files and directories
        self.p.print("Remove redundants:")
        self.p.incIndent()
        try:
            if getFileInfoListException is None:                                                        # only do removal if get-file-info-list succeeds
                fullfnList = [x[3] for x in fileInfoList]
                for fullfn in reversed(glob.glob(os.path.join(self.dataDir, "**"), recursive=True)):
                    if os.path.isfile(fullfn) and not os.path.islink(fullfn):
                        if fullfn not in fullfnList:
                            if not Util.endsWithList(fullfn, self.EXT_INFO_DICT.keys()):
                                Util.forceDelete(fullfn)
                            elif not self.persist:                                                      # note: only remove files when (not self.persist)
                                Util.forceDelete(fullfn)
                    elif os.path.isdir(fullfn) and not os.path.islink(fullfn):
                        if len(os.listdir(fullfn)) == 0:
                            os.rmdir(fullfn)
                    else:
                        # only files and directories are allowed in self.dataDir
                        Util.forceDelete(fullfn)
            self.sock.progress_changed(progressStage1 + progressStage2 + progressStage3)
        finally:
            self.p.decIndent()

        # raise the recorded error
        if getFileInfoListException is not None:
            raise getFileInfoListException

    def _get_file_info_list(self, fileInfoList, prefix=""):
        # return [(fileUrl, fileDatetime, localFn, localFullFn)]

        prefixList = []

        for entry in Util.getIndexHtmlFileList(os.path.join(self.URL, prefix)):
            # ignore robots.txt
            if prefix == "" and entry.name == "robots.txt":
                continue

            if entry.name.endswith("/"):
                # directory
                prefixList.append(os.path.join(prefix, entry.name[:-1]))
            else:
                # file
                fileUrl = os.path.join(self.URL, prefix, entry.name)
                fileName = os.path.basename(entry.name)
                localFn = os.path.join(prefix, fileName)
                localFullFn = os.path.join(self.dataDir, localFn)
                fileDatetime = Util.structTimeToDateTime(entry.modified)                                # we assume the timezone of the time on web page is UTC
                self.p.print("Found file \"%s\", last update time \"%s\"." % (localFn, fileDatetime))
                fileInfoList.append((fileUrl, fileDatetime, localFn, localFullFn))

        # linuxgame is not a "real public" website, so we make the crawling relatively slow
        Util.randomWait()

        # get file list from sub-directories
        for tprefix in prefixList:
            self._get_file_info_list(fileInfoList, tprefix)

    def _check_file_integrity(self, localFullFn):
        for ext, func in self.EXT_INFO_DICT.items():
            if localFullFn.endswith(ext):
                return func(localFullFn)

        # there exist other auxillary files (such as *.pdf)
        return False


class Util:

    @staticmethod
    def isAppImageFileValid(filepath):
        # AppImage sucks that it does not have a library or tool to simply verify a .appimage file
        # the following logic is from https://github.com/AppImage/AppImageSpec/blob/master/draft.md
        with open(filepath, "rb") as f:
            f.seek(8, 0)
            if f.read(3) != b'\x41\x49\x02':
                return False

            f.seek(0, 0)
            try:
                fileObj = elftools.elf.elffile.ELFFile(f)
                sectionObj = fileObj.get_section_by_name(".sha256_sig")
                if Util.isBufferAllZero(sectionObj.data()):
                    return True     # FIXME: really valid?
                else:
                    return True     # FIXME: really valid?
            except elftools.common.exceptions.ELFParseError:
                return False

    @staticmethod
    def isTarGzFileValid(filepath):
        try:
            subprocess.check_call(["tar", "-tzf", filepath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            return True
        except subprocess.CalledProcessError:
            return False

    @staticmethod
    def isTarBz2FileValid(filepath):
        try:
            subprocess.check_call(["tar", "-tjf", filepath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            return True
        except subprocess.CalledProcessError:
            return False

    @staticmethod
    def isTarXzFileValid(filepath):
        try:
            subprocess.check_call(["tar", "-tJf", filepath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            return True
        except subprocess.CalledProcessError:
            return False

    @staticmethod
    def is7zFileValid(filepath):
        try:
            subprocess.check_call(["7z", "t", filepath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            return True
        except subprocess.CalledProcessError:
            return False

    @staticmethod
    def isZipFileValid(filepath):
        try:
            subprocess.check_call(["unzip", "-t", filepath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            return True
        except subprocess.CalledProcessError:
            return False

    @staticmethod
    def isShFileValid(filepath):
        # FIXME
        return False

    @staticmethod
    def endsWithList(s, postfixList):
        for p in postfixList:
            if s.endswith(p):
                return True
        return False

    @staticmethod
    def isBufferAllZero(buf):
        for b in buf:
            if b != 0:
                return False
        return True

    @staticmethod
    def randomWait():
        time.sleep(random.randrange(5, 10))

    @staticmethod
    def structTimeToDateTime(src):
        return datetime(year=src.tm_year, month=src.tm_mon, day=src.tm_mday, hour=src.tm_hour, minute=src.tm_min, second=src.tm_sec)

    @staticmethod
    def ensureDir(dirname):
        if not os.path.exists(dirname):
            os.makedirs(dirname)

    @staticmethod
    def forceDelete(path):
        if os.path.islink(path):
            os.remove(path)
        elif os.path.isfile(path):
            os.remove(path)
        elif os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.exists(path):      # FIXME: device node, how to check it?
            os.remove(path)
        else:
            pass                        # path not exists, do nothing

    @staticmethod
    def getIndexHtmlFileList(url):
        while True:
            try:
                return htmllistparse.fetch_listing(url)[1]
            except requests.exceptions.RequestException:
                time.sleep(60)

    @staticmethod
    def downloadFile(url, localFile):
        while True:
            try:
                param = []
                if os.path.exists(localFile):
                    param.append("-c")
                subprocess.check_call(["wget", "--no-check-certificate", *param, "-O", localFile, url])
                break
            except subprocess.CalledProcessError:
                time.sleep(60)


class InfoPrinter:

    def __init__(self):
        self.indent = 0

    def incIndent(self):
        self.indent = self.indent + 1

    def decIndent(self):
        assert self.indent > 0
        self.indent = self.indent - 1

    def print(self, s):
        line = ""
        line += "\t" * self.indent
        line += s
        print(line)


###############################################################################

if __name__ == "__main__":
    with mirrors.plugin.ApiClient() as sock:
        try:
            Main(sock).run()
            sock.progress_changed(100)
        except Exception:
            sock.error_occured(sys.exc_info())
            raise
